from django.shortcuts import render
from .scraper import WebScraper
from .analysis import generate_wordcloud
from .models import WebContent

def scrape_view(request):
    url = 'https://example.com'  # 替换为实际要爬取的网址

    # 创建爬虫实例并爬取数据
    scraper = WebScraper(url)
    html_content = scraper.fetch_dynamic_content()
    soup = scraper.parse_html(html_content)
    text_content = scraper.extract_text(soup)
    scraper.close()

    # 生成词云
    generate_wordcloud(text_content)

    # 存储爬取的数据
    WebContent.objects.create(title="Example Title", content=text_content, url=url)

    # 渲染页面并展示内容
    return render(request, 'index.html', {'content': text_content})
